#该模块用于爬取1-7页内容，get_hotlist返回某一页的dataframe
import requests
from bs4 import BeautifulSoup
import pandas as pd

def get_html(url):
    headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36'}
    resp = requests.get(url, headers=headers)
    return resp.text


def get_data(html):
    soup = BeautifulSoup(html, 'html.parser')
    nodes = soup.find_all('div', class_='cc-cd')
    return nodes


def get_node_data(df, nodes):
    for node in nodes:
        litte_node = node.find('div', class_='cc-cd-lb')
        source = litte_node.text.strip()
        image = litte_node.find('img')['src']
        messages = node.find('div', class_='cc-cd-cb-l nano-content').find_all('a')
        for message in messages:
            if(message.find('span', class_='t')):
                content = message.find('span', class_='t').text.strip()
                data = {
                    'content': [content],
                    'url': [message['href']],
                    'source': [source],
                    'image_icon': [image]
                }
                item = pd.DataFrame(data)
                df = pd.concat([df, item], ignore_index=True)
    return df

# 爬取今日热榜 综合、科技、娱乐、社区、财经、开发、报刊板块
# 传入参数keyword指定爬取对象，分别为
#  综合、  科技、 娱乐、    社区、      财经、      开发、     报刊
# 'news','tech','ent','community','finance','developer','epaper'
def get_hotlist(keyword):
    type_dict = {1: 'news', 2: 'tech', 3: 'ent', 4: 'community', 5: 'finance', 6: 'developer', 7: 'epaper'}
    page_dict = {'news': 26, 'tech': 21, 'ent': 25, 'community': 22, 'finance': 13, 'developer': 30, 'epaper': 73}
    res = pd.DataFrame()
    for i in range(1, page_dict[type_dict[keyword]]+1):
        url = 'https://tophub.today/c/'+type_dict[keyword]+'?p='+str(i)
        html = get_html(url)
        data = get_data(html)
        res = get_node_data(res, data)   
    return res







